Model Backtesting

Code

import eia_backtesting
import pandas as pd
import plotly.graph_objects as go
import plotly.express as px
import os
import datetime
import requests
import json

Support for Torch based models not available. To enable them, install "darts", "u8darts[torch]" or "u8darts[all]" (with pip); or "u8darts-torch" or "u8darts-all" (with conda).
/opt/ai_dev_workshop/lib/python3.10/site-packages/statsforecast/core.py:27: TqdmExperimentalWarning:

Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)

Load data

Code

raw_json = open("../settings/series.json")
meta_json = json.load(raw_json)

meta_path = meta_json["meta_path"]
data_path = meta_json["data_path"]
leaderboard_path = meta_json["leaderboard_path"]

input = pd.read_csv(data_path)

input["period"] = pd.to_datetime(input["period"])

end = input.groupby(['subba'])['period'].max().min().floor(freq = "d") - datetime.timedelta(hours = 1)

print(end)


input.head()

2024-06-14 23:00:00

	period	subba	subba-name	parent	parent-name	value	value-units
0	2018-07-01 08:00:00	PGAE	Pacific Gas and Electric	CISO	California Independent System Operator	12522.0	megawatthours
1	2018-07-01 09:00:00	PGAE	Pacific Gas and Electric	CISO	California Independent System Operator	11745.0	megawatthours
2	2018-07-01 10:00:00	PGAE	Pacific Gas and Electric	CISO	California Independent System Operator	11200.0	megawatthours
3	2018-07-01 11:00:00	PGAE	Pacific Gas and Electric	CISO	California Independent System Operator	10822.0	megawatthours
4	2018-07-01 12:00:00	PGAE	Pacific Gas and Electric	CISO	California Independent System Operator	10644.0	megawatthours

Code

p = px.line(input, x="period", y="value", color="subba",
 labels={
                     "period": "Time",
                     "value": "MegaWattHours",
                     "subba": "Sub-Region"
                 },
title = "California Hourly Demand By Operating Provider")
p.show()

Code

freq = meta_json["backtesting"]["freq"]
h = meta_json["backtesting"]["h"]
overlap = meta_json["backtesting"]["overlap"]
tags = meta_json["backtesting"]["tags"]
experiment_name = meta_json["backtesting"]["experiment_name"]
mlflow_path = meta_json["backtesting"]["mlflow_path"]
p = meta_json["backtesting"]["p"]
pi = meta_json["backtesting"]["pi"]
quantiles = meta_json["backtesting"]["quantiles"]
seed = meta_json["backtesting"]["seed"]


params1 = meta_json["backtesting"]["models"]["model1"]
params2 = meta_json["backtesting"]["models"]["model2"]
params3 = meta_json["backtesting"]["models"]["model3"]
params4 = meta_json["backtesting"]["models"]["model4"]
params5 = meta_json["backtesting"]["models"]["model5"]
params6 = meta_json["backtesting"]["models"]["model6"]
params7 = meta_json["backtesting"]["models"]["model7"]

params = [params1, params2, params3, params4, params5, params7]

for i in range(len(params)):
    params[i]["h"] = h
    params[i]["freq"] = freq
    params[i]["quantiles"] = quantiles
    params[i]["pi"] = pi
    params[i]["seed"] = seed

Code

series = pd.DataFrame(meta_json["series"])

backtesting_output = {}
for index, row in series.iterrows():
    print("Start backtesting for " + row['subba_id'])
    df = None
    df = input[input["subba"] == row['subba_id']]
    df = df.sort_values(by = ["period"])
    par = params
    for i in range(len(par)):
        par[i]["subba"] = row['subba_id']

    backtesting_output[row['subba_id']] = eia_backtesting.backtesting(input = df, 
            partitions=p, 
            overlap = overlap, 
            h = h, 
            params = par,
            experiment_name = experiment_name + "_" + row['subba_id'],
            mlflow_path = mlflow_path,
            overwrite = True,
            tags = tags)

Code

leaderboard = None
best = None

for index, row in series.iterrows():
    subba = row["subba_id"]
    print(subba)
    print(backtesting_output[subba].leaderboard)
    leader_subba = backtesting_output[subba].leaderboard
    leader_subba["subba"] = subba

    if leaderboard is None:
        leaderboard = backtesting_output[subba].leaderboard
        best =  leader_subba[leader_subba["mape"] == leader_subba["mape"].min()]
    else:
        leaderboard_temp = backtesting_output[subba].leaderboard
        leaderboard = leaderboard._append(leaderboard_temp)
        best = best._append(leader_subba[leader_subba["mape"] == leader_subba["mape"].min()])

    
#best = best[["subba", "model","model_label" ,"mape", "rmse"]] 
best.to_csv(leaderboard_path, index = False)   

best

PGAE
  forecast_label model_label                  model      mape         rmse  \
0     2024-06-14      model1  LinearRegressionModel  0.089334  1365.539586   
1     2024-06-14      model2  LinearRegressionModel  0.090086  1374.423744   
2     2024-06-14      model3  LinearRegressionModel  0.096589  1456.745612   
3     2024-06-14      model4  LinearRegressionModel  0.091571  1415.877250   
4     2024-06-14      model5  LinearRegressionModel  0.092669  1404.666182   
5     2024-06-14      model7               XGBModel  0.091234  1333.922406   

   coverage                                           comments  
0  0.725000  LM model with lags, training with 2 years of h...  
1  0.729167  LM model with lags, training with 3 years of h...  
2  0.537500                                 Model 2 with lag 1  
3  0.725000                       Model 1 with additional lags  
4  0.725000                       Model 1 with additional lags  
5  0.670833                                  XGBoost with lags  
SCE
  forecast_label model_label                  model      mape         rmse  \
0     2024-06-14      model1  LinearRegressionModel  0.052259   824.558770   
1     2024-06-14      model2  LinearRegressionModel  0.051878   817.262735   
2     2024-06-14      model3  LinearRegressionModel  0.077653  1139.373683   
3     2024-06-14      model4  LinearRegressionModel  0.068839  1073.844087   
4     2024-06-14      model5  LinearRegressionModel  0.071172  1121.572839   
5     2024-06-14      model7               XGBModel  0.077308  1153.257120   

   coverage                                           comments  
0  0.929167  LM model with lags, training with 2 years of h...  
1  0.929167  LM model with lags, training with 3 years of h...  
2  0.695833                                 Model 2 with lag 1  
3  0.887500                       Model 1 with additional lags  
4  0.862500                       Model 1 with additional lags  
5  0.775000                                  XGBoost with lags  
SDGE
  forecast_label model_label                  model      mape        rmse  \
0     2024-06-14      model1  LinearRegressionModel  0.083502  187.714005   
1     2024-06-14      model2  LinearRegressionModel  0.083141  188.657150   
2     2024-06-14      model3  LinearRegressionModel  0.131489  266.710639   
3     2024-06-14      model4  LinearRegressionModel  0.125430  257.151173   
4     2024-06-14      model5  LinearRegressionModel  0.123442  251.715517   
5     2024-06-14      model7               XGBModel  0.129947  261.224281   

   coverage                                           comments  
0  0.933333  LM model with lags, training with 2 years of h...  
1  0.933333  LM model with lags, training with 3 years of h...  
2  0.770833                                 Model 2 with lag 1  
3  0.879167                       Model 1 with additional lags  
4  0.887500                       Model 1 with additional lags  
5  0.875000                                  XGBoost with lags  
VEA
  forecast_label model_label                  model      mape       rmse  \
0     2024-06-14      model1  LinearRegressionModel  0.100615  18.513913   
1     2024-06-14      model2  LinearRegressionModel  0.100654  18.651437   
2     2024-06-14      model3  LinearRegressionModel  0.150168  30.266325   
3     2024-06-14      model4  LinearRegressionModel  0.127715  25.427010   
4     2024-06-14      model5  LinearRegressionModel  0.129275  25.454507   
5     2024-06-14      model7               XGBModel  0.114773  22.695727   

   coverage                                           comments  
0  0.887500  LM model with lags, training with 2 years of h...  
1  0.833333  LM model with lags, training with 3 years of h...  
2  0.787500                                 Model 2 with lag 1  
3  0.850000                       Model 1 with additional lags  
4  0.850000                       Model 1 with additional lags  
5  0.858333                                  XGBoost with lags

	forecast_label	model_label	model	mape	rmse	coverage	comments	subba
0	2024-06-14	model1	LinearRegressionModel	0.089334	1365.539586	0.725000	LM model with lags, training with 2 years of h...	PGAE
1	2024-06-14	model2	LinearRegressionModel	0.051878	817.262735	0.929167	LM model with lags, training with 3 years of h...	SCE
1	2024-06-14	model2	LinearRegressionModel	0.083141	188.657150	0.933333	LM model with lags, training with 3 years of h...	SDGE
0	2024-06-14	model1	LinearRegressionModel	0.100615	18.513913	0.887500	LM model with lags, training with 2 years of h...	VEA

Code

print(leaderboard)

  forecast_label model_label                  model      mape         rmse  \
0     2024-06-14      model1  LinearRegressionModel  0.089334  1365.539586   
1     2024-06-14      model2  LinearRegressionModel  0.090086  1374.423744   
2     2024-06-14      model3  LinearRegressionModel  0.096589  1456.745612   
3     2024-06-14      model4  LinearRegressionModel  0.091571  1415.877250   
4     2024-06-14      model5  LinearRegressionModel  0.092669  1404.666182   
5     2024-06-14      model7               XGBModel  0.091234  1333.922406   
0     2024-06-14      model1  LinearRegressionModel  0.052259   824.558770   
1     2024-06-14      model2  LinearRegressionModel  0.051878   817.262735   
2     2024-06-14      model3  LinearRegressionModel  0.077653  1139.373683   
3     2024-06-14      model4  LinearRegressionModel  0.068839  1073.844087   
4     2024-06-14      model5  LinearRegressionModel  0.071172  1121.572839   
5     2024-06-14      model7               XGBModel  0.077308  1153.257120   
0     2024-06-14      model1  LinearRegressionModel  0.083502   187.714005   
1     2024-06-14      model2  LinearRegressionModel  0.083141   188.657150   
2     2024-06-14      model3  LinearRegressionModel  0.131489   266.710639   
3     2024-06-14      model4  LinearRegressionModel  0.125430   257.151173   
4     2024-06-14      model5  LinearRegressionModel  0.123442   251.715517   
5     2024-06-14      model7               XGBModel  0.129947   261.224281   
0     2024-06-14      model1  LinearRegressionModel  0.100615    18.513913   
1     2024-06-14      model2  LinearRegressionModel  0.100654    18.651437   
2     2024-06-14      model3  LinearRegressionModel  0.150168    30.266325   
3     2024-06-14      model4  LinearRegressionModel  0.127715    25.427010   
4     2024-06-14      model5  LinearRegressionModel  0.129275    25.454507   
5     2024-06-14      model7               XGBModel  0.114773    22.695727   

   coverage                                           comments subba  
0  0.725000  LM model with lags, training with 2 years of h...  PGAE  
1  0.729167  LM model with lags, training with 3 years of h...  PGAE  
2  0.537500                                 Model 2 with lag 1  PGAE  
3  0.725000                       Model 1 with additional lags  PGAE  
4  0.725000                       Model 1 with additional lags  PGAE  
5  0.670833                                  XGBoost with lags  PGAE  
0  0.929167  LM model with lags, training with 2 years of h...   SCE  
1  0.929167  LM model with lags, training with 3 years of h...   SCE  
2  0.695833                                 Model 2 with lag 1   SCE  
3  0.887500                       Model 1 with additional lags   SCE  
4  0.862500                       Model 1 with additional lags   SCE  
5  0.775000                                  XGBoost with lags   SCE  
0  0.933333  LM model with lags, training with 2 years of h...  SDGE  
1  0.933333  LM model with lags, training with 3 years of h...  SDGE  
2  0.770833                                 Model 2 with lag 1  SDGE  
3  0.879167                       Model 1 with additional lags  SDGE  
4  0.887500                       Model 1 with additional lags  SDGE  
5  0.875000                                  XGBoost with lags  SDGE  
0  0.887500  LM model with lags, training with 2 years of h...   VEA  
1  0.833333  LM model with lags, training with 3 years of h...   VEA  
2  0.787500                                 Model 2 with lag 1   VEA  
3  0.850000                       Model 1 with additional lags   VEA  
4  0.850000                       Model 1 with additional lags   VEA  
5  0.858333                                  XGBoost with lags   VEA

PGAE

Code

 eia_backtesting.plot_score(score = backtesting_output["PGAE"].leaderboard, type = "box")

Code

eia_backtesting.plot_score(score = backtesting_output["PGAE"].score, type = "line")

SCE

Code

 eia_backtesting.plot_score(score = backtesting_output["SCE"].leaderboard, type = "box")

Code

eia_backtesting.plot_score(score = backtesting_output["SCE"].score, type = "line")

SDGE

Code

 eia_backtesting.plot_score(score = backtesting_output["SDGE"].leaderboard, type = "box")

Code

eia_backtesting.plot_score(score = backtesting_output["SDGE"].score, type = "line")

VEA

Code

 eia_backtesting.plot_score(score = backtesting_output["VEA"].leaderboard, type = "box")

Code

eia_backtesting.plot_score(score = backtesting_output["VEA"].score, type = "line")